
package com.kinth.work.huicong2;
import java.io.File;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.util.*;


import org.apache.log4j.xml.DOMConfigurator;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;



import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Request;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.scheduler.PriorityScheduler;

/**
 * 
 * <p>
 * title: 抓取慧聪网，4月28日更新。解决之前使用连接池的问题。觉得不用链接池。
 * 1. 因为总的类别有832个。所以不能存到内存里面，只能存到数据库里面。所以使用的方法就是先吧总的数据从到huicong_url1这个
 *    表里面。然后后面每次取一个。因为这个表已经存在，就在下面的程序里面吧这部分代码注释掉了。如果需要，可以去掉注释。
 * 2. 目前采用传入一个类别的网址，然后按照优先级迭代。
 * 3. 可是貌似速度太快会被屏蔽。这里还需要在看看。但是貌似是会反扒屏蔽ip的。之前没有遇到过。
 * 4. 当然，反扒只限于具体网页。
 * **********************************************************************
 * 5月8号更新：
 * 1. 目前已经抛弃c3p0.因为报错的问题解决不了。也许更新了c3p0的版本之后可以解决。但是觉得自己已经没有必要试了。根据
 *    需求，大概几天也只有一次读取数据库。这样根本不需要连接池了。直接上链接即可。
 * </p>
 * <p>
 * 
 * </p>
 * 
 * @author kaikai
 * @2017年4月1日
 */
public class HuiCong implements PageProcessor
{
	private Site site = Site.me().setUserAgent("Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 "
			+ "(KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36").setRetryTimes(3).setSleepTime(100);

	public static final String BIG_URL_ENTRY = "http://info.b2b.hc360.com/list/company.shtml";
	public static final String BIG_URL = "http://s.hc360.com/\\?w=[\\s\\S]*?&mychannel=enterprise";
    public static final String BIG_URL2 =  "http://s.hc360.com/\\?w=[\\s\\S]*?&mychannel=enterprise&ee=[0-9]+";
	public  static final String COMPANY_URL = "http://[a-zA-Z0-9]+.b2b.hc360.com/";
	
	public static final int NUM_ONCE = 1;//每次从huicong1里面取出的url数目
	//添加每个链接的延迟，为了不被ip屏蔽
    public static final int SLEEP_TIME = 5000;
    
    //定位文件的地址
   
    
    protected Logger logger = LoggerFactory.getLogger(getClass());
    
	@Override
	public void process(Page page)
	{
		
		if(page.getUrl().regex(BIG_URL_ENTRY).match())
		{
			//这是获得首页的ip并存到数据库公后面使用。因为有830个url，如果全部存到内存里面会太多了。但是目前
			//数据库里面huicong_url1这张表是满的，也就不需要再运行这个了。所以暂时注销。如果以后需要，可以继续。
			
			try
			{
				//getListUrlEntry(page);
			} catch (Exception e)
			{
				// TODO Auto-generated catch block
				e.printStackTrace();
			}
			
			
		}else if(page.getUrl().regex(BIG_URL2).match())
		{

			
			try{
				processBigUrl(page);
			}catch(Exception e)
			{
				logger.error("获得公司链接出错:  "  + " error", e);
			}
			
		}else if(page.getUrl().regex(BIG_URL).match())
		{
			
			try{
				processBigUrl0(page);
			}catch(Exception e)
			{
				logger.error("获得公司列表链接出错:  "  + " error", e);
			}
			
		}else if(page.getUrl().regex(COMPANY_URL).match())
		{

			
			try{
				//为了克服反扒，一秒一次。
				Thread.sleep(1000);
				getInfo(page);
			}catch(Exception e)
			{
				logger.error("获得公司信息出错:  "  + " error", e);
			}
			
		}


	}

	//获得公司列表链接入口的链接。
	public void getListUrlEntry(Page page) throws Exception
	{
		ArrayList<String> nameList = new ArrayList<String>();
		nameList = (ArrayList<String>) page.getHtml().xpath("//body//div[@class='indMarketClassBox']//a/text()").all();

		for (String name : nameList)
		{
			if (name.equals("更多"))
			{
				continue;
			}
			try
			{
				String nameTemp = name;
				name = URLEncoder.encode(name, "gb2312");
				String url = "http://s.hc360.com/?w=" + name + "&mychannel=enterprise";
				UrlHuiCong huiCong = new UrlHuiCong(url, nameTemp);
                
				dbUtil.insertUrl(huiCong, "huicong_url1");
				
				logger.info("获取公司列表链接的入口："+url);

			} catch (UnsupportedEncodingException e)
			{
				// TODO Auto-generated catch block
				e.printStackTrace();
			} catch (IOException e)
			{
				// TODO Auto-generated catch block
				e.printStackTrace();
			}

		}
	}
	//处理公司页面
	public void getInfo(Page page) throws Exception
	{

		
		// 公司名称
				String name = page.getHtml().xpath("//body/div[@id='innerBgBox']"
						+ "/div[@id='mainbox']//div[@class='columnbox colrel']" + "//div[@class='comInfogo']/ul/li[1]/a/text()")
						.toString();

				String people = null;
				String address = null;
				String phone1 = null;
				String phone2 = null;
				String phone3 = null;
				String qq = null;
				String introduction = null;
				String product = null;
				if (name != null)
				{
					// 产品
					product = page.getHtml().xpath("//head/title/text()").toString();

					// 联系人
					people = page.getHtml()
							.xpath("//body/div[@id='innerBgBox']" + "/div[@id='mainbox']//div[@class='columnbox colrel']"
									+ "//div[@class='comInfogo']/ul/li[2]/span/a[1]/text()")
							.toString();

					// 地址
					address = page.getHtml()
							.xpath("//body/div[@id='innerBgBox']" + "/div[@id='mainbox']//div[@class='columnbox colrel']"
									+ "//div[@class='archives colrel']/div[3]/text()")
							.toString();

					// 联系qq
					qq = page.getHtml().xpath("//body/div[@id='innerBgBox']"
							+ "/div[@id='mainbox']//div[@class='columnbox colrel']" + "//div[@class='comInfogo']//li[2]/a")
							.links().toString();

					//System.out.println("测试qq: "+qq);
					// 电话
					String phoneTemp = page.getHtml()
							.xpath("//body/div[@id='innerBgBox']/div[@id='mainbox']//div[@class='txtb']"
									+ "//div[@class='content-inner']/div[2]/span/text()")
							.toString();
					phone1 = page.getHtml().xpath("//body/div[@id='innerBgBox']/div[@id='mainbox']//div[@class='txtb']"
							+ "//div[@class='content-inner']/div[2]/em/text()").toString();
					if (phone1 != null)
					{
						phone1 = phoneTemp + phone1;
					}

					// 手机
					phoneTemp = page.getHtml().xpath("//body/div[@id='innerBgBox']/div[@id='mainbox']//div[@class='txtb']"
							+ "//div[@class='content-inner']/div[3]/span/text()").toString();
					phone2 = page.getHtml().xpath("//body/div[@id='innerBgBox']/div[@id='mainbox']//div[@class='txtb']"
							+ "//div[@class='content-inner']/div[3]/em/text()").toString();
					if (phone2 != null)
					{
						phone2 = phoneTemp + phone2;
					}

					// 传真
					phoneTemp = page.getHtml().xpath("//body/div[@id='innerBgBox']/div[@id='mainbox']//div[@class='txtb']"
							+ "//div[@class='content-inner']/div[4]/span/text()").toString();
					phone3 = page.getHtml().xpath("//body/div[@id='innerBgBox']/div[@id='mainbox']//div[@class='txtb']"
							+ "//div[@class='content-inner']/div[4]/em/text()").toString();
					if (phone3 != null)
					{
						phone3 = phoneTemp + phone3;
					}

					// 公司介绍
					introduction = page.getHtml().xpath("//body/div[@id='innerBgBox']" + "/div[@id='mainbox']"
							+ "//div[@class='contentbox']" + "//div[@class='cAbout']/text()").toString();

					// 此处是调用phantomJS来提取动态加载的qq。但是因为目前慧聪网此处加载的qq是网站的客服，就是全部都一样。所以先
					// 注释掉整个if
					// if(qq == null)
					// {
					// //companyServiceMod
					// try
					// {
					// qq = JSUtil.getParseredHtml2(urlNow);
					// //System.out.println("测试： "+qq);
					// String regex = "<a[\\s\\S]*?>";
					// Pattern pa = Pattern.compile(regex, Pattern.DOTALL);
					// Matcher ma = pa.matcher(qq);
					// if(ma.find())
					// {
					// qq = ma.group(0);
					// }
					//
					// } catch (IOException e)
					// {
					// e.printStackTrace();
					// }
					//
					// }
					if (qq != null)
					{
						String[] tempqq = qq.split("&");
						if (tempqq.length >= 2)
						{
							String tempqq2 = tempqq[1];

							tempqq = tempqq2.split("=");
							if (tempqq.length >= 2)
							{
								tempqq2 = tempqq[1];

								qq = tempqq2;
							} else
							{
								qq = null;
							}

						} else
						{
							qq = null;
						}
					}
                    String urlNow = page.getUrl().toString() ;
					String industry = (String)page.getRequest().getExtra("industry");
					
					logger.info("获取公司信息："+name);
					
					page.putField("name", name);
					page.putField("product", product);
					page.putField("address", address);
					page.putField("people", people);
					page.putField("qq", qq);
					page.putField("introduction", introduction);
					page.putField("phone1", phone1);
					page.putField("phone2", phone2);
					page.putField("phone3", phone3);
					page.putField("industry", industry);
					page.putField("url", urlNow);
				} else
				{
					name = page.getHtml()
							.xpath("//body/div[@class='newWrap']" + "//div[@class='proBox25left']/div[@class='columnbox']"
									+ "/div[@class='leftBoxCon qyArchives']/h2/a/text()")
							.toString();
					System.out.println("name2: " + name);
					if (name != null)
					{
						// 联系人
						people = page.getHtml()
								.xpath("//body/div[@class='newWrap']" + "//div[@class='proBox25left']/div[@class='columnbox']"
										+ "/div[@class='leftBoxCon qyArchives']/div[@class='ArchivesList']" + "//p/a/text()")
								.toString();

						// 地址
						address = page.getHtml()
								.xpath("//body/div[@class='newWrap']" + "//div[@class='proBox25left']/div[@class='columnbox']"
										+ "/div[@class='leftBoxCon qyArchives']/div[@class='ArchivesList']"
										+ "/ul/li[3]/div/text()")
								.toString();

						qq = page.getHtml()
								.xpath("//body/div[@class='newWrap']" + "//div[@class='proBox25left']/div[@class='columnbox']"
										+ "/div[@class='leftBoxCon qyArchives']/div[@class='ArchivesList']"
										+ "//div[@class='aRight']/a")
								.links().toString();

						// 此处是调用phantomJS来提取动态加载的qq。但是因为目前慧聪网此处加载的qq是网站的客服，就是全部都一样。所以先
						// 注释掉整个if
						// if(qq == null)
						// {
						// try
						// {
						// qq = JSUtil.getParseredHtml2(urlNow);
						// String regex = "<a[\\s\\S]*?>";
						// Pattern pa = Pattern.compile(regex, Pattern.DOTALL);
						// Matcher ma = pa.matcher(qq);
						// if(ma.find())
						// {
						// qq = ma.group(0);
						// }
						//
						// } catch (IOException e)
						// {
						// e.printStackTrace();
						// }
						//
						// }
						if (qq != null)
						{
							String[] tempqq = qq.split("&");
							if (tempqq.length >= 2)
							{
								String tempqq2 = tempqq[1];

								tempqq = tempqq2.split("=");
								if (tempqq.length >= 2)
								{
									tempqq2 = tempqq[1];

									qq = tempqq2;
								} else
								{
									qq = null;
								}

							} else
							{
								qq = null;
							}

						}

						String phoneTemp = page.getHtml()
								.xpath("//body/div[@class='newWrap']"
										+ "//div[@class='proBox25left']//div[@class='leftBoxCon ContactBox']"
										+ "/ul/li[2]/span[1]/text()")
								.toString();
						phone1 = page.getHtml()
								.xpath("//body/div[@class='newWrap']"
										+ "//div[@class='proBox25left']//div[@class='leftBoxCon ContactBox']"
										+ "/ul/li[2]/span[2]/text()")
								.toString();
						phone1 = phoneTemp + phone1;

						phoneTemp = page.getHtml()
								.xpath("//body/div[@class='newWrap']"
										+ "//div[@class='proBox25left']//div[@class='leftBoxCon ContactBox']"
										+ "/ul/li[3]/span[1]/text()")
								.toString();
						phone2 = page.getHtml()
								.xpath("//body/div[@class='newWrap']"
										+ "//div[@class='proBox25left']//div[@class='leftBoxCon ContactBox']"
										+ "/ul/li[3]/span[2]/text()")
								.toString();
						phone2 = phoneTemp + phone2;

						phoneTemp = page.getHtml()
								.xpath("//body/div[@class='newWrap']"
										+ "//div[@class='proBox25left']//div[@class='leftBoxCon ContactBox']"
										+ "/ul/li[4]/span[1]/text()")
								.toString();
						phone3 = page.getHtml()
								.xpath("//body/div[@class='newWrap']"
										+ "//div[@class='proBox25left']//div[@class='leftBoxCon ContactBox']"
										+ "/ul/li[4]/span[2]/text()")
								.toString();
						if (phone3 != null)
						{
							phone3 = phoneTemp + phone3;
						}

						introduction = page.getHtml()
								.xpath("//body/div[@class='newWrap']" + "//div[@class='proBox75Rig']/div[@class='columnbox']"
										+ "//div[@class='conpanyBoxSingle']/p/text()")
								.toString();
						if (introduction == null)
						{
							introduction = page.getHtml()
									.xpath("//body/div[@class='newWrap']"
											+ "//div[@class='proBox75Rig']/div[@class='columnbox']"
											+ "//div[@class='conpanyBox']/p/text()")
									.toString();
						}
						product = page.getHtml().xpath("//head/meta[@name='description']").toString();
						// 只用取一次即可
						String urlNow = page.getUrl().toString();
						String industry = (String)page.getRequest().getExtra("industry");
						
						logger.info("获取公司信息："+name);
						
						page.putField("name", name);
						page.putField("product", product);
						page.putField("address", address);
						page.putField("people", people);
						page.putField("qq", qq);
						page.putField("introduction", introduction);
						page.putField("phone1", phone1);
						page.putField("phone2", phone2);
						page.putField("phone3", phone3);
						page.putField("industry", industry);
						page.putField("url", urlNow);
					}
				}


	}
	//得到全部页数，然后转换得到全部列表页
	public void processBigUrl0(Page page) throws Exception
	{
		
		String pageAmount = page.getHtml()
				.xpath("/html/body/div[@class='s-layout']" + "/div[@class='s-mod-bcrumbs']/span/em/text()").toString();

		if (pageAmount != null)
		{
			int pageAmount2 = Integer.parseInt(pageAmount);
			int pageAmount3 = pageAmount2 / 40 + 1;

			String urlNow = page.getUrl().toString();
			String industry = (String) page.getRequest().getExtra("industry");
		

			if (pageAmount3 > 100)
			{
				pageAmount3 = 100;
			}
			//为了测试，先用5来代替
			for (int i = 1; i <= pageAmount3; i++)
			//for (int i = 2; i <= 5; i++)
			{
				String urlnext = urlNow + "&ee=" + i;
				
				Request request = new Request(urlnext).putExtra("industry", industry).setPriority(2);
				page.addTargetRequest(request); 

				logger.info("获取公司列表的链接: "+urlnext);
			}
		}
		
	}
	//处理列表页
	public void processBigUrl(Page page) throws Exception
	{
		ArrayList<String> companyList = new ArrayList<String>();
		companyList = (ArrayList<String>) page.getHtml()
				.xpath("//body/div[@class='s-layout']" + "/div[@class='s-mod-main']/div[@class='cont-left']"
						+ "/div[@class='col']/div[@class='contbox']//h3/a")
				.links().all();

		String industry = (String)page.getRequest().getExtra("industry");
		
		for (String url : companyList)
		{
			
			Request request = new Request(url).putExtra("industry", industry).setPriority(3);
			page.addTargetRequest(request); 
			logger.info("获取公司的链接: "+url);
		}
	}
	
	@Override
	public Site getSite()
	{
		return site;
	}
    
	// 一个代替main函数的函数
	public static void runSpider() throws Exception
	{
		//不能通过
		//把url也存起来
		
		System.out.println("爬取类别链接开始");
		HuiCong huiCong = new HuiCong();
		Spider spider = Spider.create(huiCong);
		
		
		spider
		.setScheduler(new PriorityScheduler())
		.addPipeline(new DataPipeline())
		;
		
		int s = FileUtil.getBigUrl();
		
		int rank = 1;
		//此处要小心，s必须存在。
		rank = s;
		//记录每次的位置。第一次从文件里面读入。
		
		while(true)
		{
			ArrayList<UrlHuiCong> urlList = dbUtil.queryUrl("huicong_url1", rank, NUM_ONCE);
			UrlHuiCong urlHuiCong = null;
			String url = null;
			String industry = null;

			if(urlList.size() == 1)
			{
				urlHuiCong = urlList.get(0);
				url = urlHuiCong.getUrl();
				industry = urlHuiCong.getIndustry();
			}
			Request request = new Request(url).putExtra("industry", industry).setPriority(1);
			
			//run不能写到上面。
			spider.addRequest(request ).run();

			while(spider.getStatus()!=Spider.Status.Stopped)
			{
				try
				{
					Thread.sleep(100 * 1000);
				} catch (InterruptedException e)
				{
					// TODO Auto-generated catch block
					e.printStackTrace();
				}
			}
			
			rank++;
			FileUtil.writeBigUrl(rank);
		}
	
	}

	public static void main(String[] args)
	{
		DOMConfigurator.configure(System.getProperty("user.dir")+File.separator+"log4j.xml");
		try
		{
			runSpider();
		} catch (Exception e)
		{
			// TODO Auto-generated catch block
			e.printStackTrace();
		}

	}

	
}